from anole.core.checkpoint import ICheckable
from anole.utils.patterns.Singleton import Singleton
import threading,md5
class ContentSeen(Singleton,ICheckable):
	def __init__(self):
		self._seen={}
		self._lock=threading.Lock()
		ICheckable.__init__(self)
	def is_seen(self,strHTML,insert=True):
		fgp=md5.new(strHTML).digest()
		self._lock.acquire()
		try:
			if self._seen.has_key(fgp):
				return True
			else:	
			
				if insert:
					self._seen[fgp]=1
				return False
			
		finally:
			self._lock.release()
	def save(self,taskName=''):
		self._lock.acquire()
		try:
			f=open(taskName+'-contentseen.bak','wb')
			size=len(self._seen.keys())
			f.write(str(size)+'\n')
			buf=[]
			i=0
			keys=self._seen.keys()
			for key in keys:
		
				buf.append(key)
				i+=1
				if len(buf)>1000 or i>=size:
					#print i
					#s= ''.join(buf)
					#print len(s)
					f.write(''.join(buf))
					buf=[]
			f.close()
		finally:
			self._lock.release()
		
	def load(self,taskName=''):
		try:
			f=open(taskName+'-contentseen.bak','rb')
		except IOError:
			return
		size=int(f.readline())
		#print 'load size is: ',size
		i=0
		while i<size:
			#print 'load loop'
			buf=f.read(16000)
			num=len(buf)/16
			for x in xrange(num):
				contentMD5=buf[x*16:x*16+16]
				self._seen[contentMD5]=1
			i+=num
		f.close()
